Pi Stocks

Work Distribution

Project group picture

Thomas Sand

tms265@cornell.edu

Designed display and program communication.

Shuhan Mao

sm2376@cornell.edu

Trained tensorflow models and wrote the predicting function.

Code Appendix

# Coordinates Display and Prediction Reader
#! /usr/bin/bash

FILE=FIFO
if [ ! -p "$FILE" ]; then
    mkfifo FIFO
fi

echo "kickstart" > FIFO &
./listener.py &
./stock_tracker.py

pkill -P $$

              

# Prediction Sender (sender.py)
#! /usr/bin/python3

import os
import sys
import signal
import requests
import time
import csv
import socket
import numpy as np
import pickle
import tensorflow as tf
from tensorflow.keras.models import load_model


HOST = "10.49.66.69"  # The server's hostname or IP address
PORT = 65432  # The port used by the server

def handler(signum, frame):
    os.system('clear')
    sys.exit()

def get_model(model_path, scaler_path, stock_price): 
    #stock_price is a 2-D array
    #model = load_model(model_path)  
    model = pickle.load(open(model_path,'rb'))
    
    scaler = pickle.load(open(scaler_path,'rb'))
    sc = scaler.transform(stock_price)

    estimate = model.predict(sc)
    estimate = scaler.inverse_transform(estimate)
    return estimate

def getClose(name_string): # Return yesterday's closing price
    url = "https://twelve-data1.p.rapidapi.com/time_series"

    querystring = {"symbol":name_string,"interval":"1day","outputsize":"1","format":"json"}

    headers = {
        "X-RapidAPI-Host": "twelve-data1.p.rapidapi.com",
        "X-RapidAPI-Key": ""
    }

    response = requests.request("GET", url, headers=headers, params=querystring)
    converted = response.json()
    price_string = converted['values'][0]['close'].rstrip("0")
    return round(float(price_string), 2)


def csv_to_data():
    data = []
    with open('names_prices.txt','r') as FD:
        reader = csv.reader(FD) 
        for row in reader:
           data.append(row[0])
           data.append(row[1])

    FD.close()
    return data

signal.signal(signal.SIGINT, handler)
names_and_prices = csv_to_data()
stock_names = names_and_prices[::2]
name_index = 0

while True:

    last_close = getClose(stock_names[name_index])
    close_price = np.reshape(last_close, (-1, 1))
    scalerfile = "model_files/" + stock_names[name_index] + "_scaler"
    modelfile = "model_files/My_"+ stock_names[name_index] + "_model.sav"
    
    forcast = get_model(modelfile, scalerfile, close_price)
    forcast = forcast[0][0]
    forcast_str = str(round(forcast,2))
    forcast_str = "Est. Close: " + stock_names[name_index] + " " + forcast_str 

    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.connect((HOST, PORT))
        s.sendall(forcast_str.encode('utf-8'))
        s.close()
    
    name_index += 1
    if name_index >= len(stock_names):
        name_index = 0
    
    time.sleep(180)
              

# Display Handler (stock_tracker.py)
#! /usr/bin/python3

import time
from datetime import datetime
import requests
import json
import os
import RPi.GPIO as GPIO
import pygame
import sys
import numpy as np
import csv
from pygame.locals import *

os.nice(30)

os.putenv('SDL_VIDEODRIVER','fbcon')
os.putenv('SDL_FBDEV','/dev/fb1')
os.putenv('SDL_MOUSEDRV','TSLIB')
os.putenv('SDL_MOUSEDEV','/dev/input/touchscreen')

def GPIO27_callback(channel): # Triggered by physical exit button
    global EXIT
    EXIT = True

def getPrice(name_string): # Returns string value for price of valid NASDAQ stock id (ex. TXN)
    url = "https://twelve-data1.p.rapidapi.com/price"

    querystring = {"symbol": name_string,"format":"json","outputsize":"30"}

    headers = {
        "X-RapidAPI-Host": "twelve-data1.p.rapidapi.com",
        "X-RapidAPI-Key": ""
    }

    response = requests.request("GET", url, headers=headers, params=querystring)
    converted = response.json()
    return converted['price'].rstrip("0")

def getChart(name_string): # Gets 100 points from 1month, IDK if should be adjustable
    url = "https://twelve-data1.p.rapidapi.com/time_series"

    querystring = {"symbol":"AMZN","interval":"1day","outputsize":"30","format":"json"}

    headers = {
        "X-RapidAPI-Host": "twelve-data1.p.rapidapi.com",
        "X-RapidAPI-Key": ""
    }

    response = requests.request("GET", url, headers=headers, params=querystring)
    fieldnames = ['datetime', 'open', 'high', 'low', 'close', 'volume']
    json_data = response.json()
    filename = name_string + '_data.csv'
    with open(filename, 'w') as SDcsv:
        writer = csv.DictWriter(SDcsv, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(json_data['values'])
        
    SDcsv.close()


def data_to_csv(n, p):
    with open('names_prices.txt','w') as FD:
        writer = csv.writer(FD)
        for x in range(len(n)):
            data = [n[x], p[x]]
            writer.writerow(data)

    FD.close()

def csv_to_data():
    data = []
    with open('names_prices.txt','r') as FD:
        reader = csv.reader(FD) 
        for row in reader:
           data.append(row[0])
           data.append(row[1])

    FD.close()
    return data


GPIO.setmode(GPIO.BCM) # set as broadcom
GPIO.setup(27, GPIO.IN, pull_up_down=GPIO.PUD_UP) # set GPIO27 to pullup
GPIO.add_event_detect(27, GPIO.FALLING, callback=GPIO27_callback)

EXIT = False
SCROLL = True
PRINT_1 = True
PRINT_2 = False

pygame.init()
pygame.mouse.set_visible(False)

# Background
WHITE = 255,255,255
BLACK = 0,0,0
RED = 255,0,0
GREEN = 55,235,52
width = 320
height = 240
SCREEN = pygame.display.set_mode((width,height))

# Words
FONT = pygame.font.Font(None,160)
small_FONT = pygame.font.Font(None,80)

class window:
    addElement = True
    def __init__(self, text, color):
        self.text = text
        self.surface = FONT.render(text, True, color)
        self.loc = [(width + int(self.surface.get_width()/2)), 80]
        self.rect = self.surface.get_rect(center = self.loc)
        self.trigger = width - int(self.surface.get_width()/2)
        self.EXIT = -1*int(self.surface.get_width()/2)

    def move(self):
        self.loc[0] -= 1

    def update(self):         
        self.rect = self.surface.get_rect(center = self.loc)
        SCREEN.blit(self.surface,self.rect) 

class subwindow:
    addElement = True
    def __init__(self, text):
        self.text = text
        self.surface = small_FONT.render(text, True, WHITE)
        self.loc = [(width + int(self.surface.get_width()/2)), 200]
        self.rect = self.surface.get_rect(center = self.loc)
        #self.trigger = width - int(self.surface.get_width()/2)
        self.EXIT = -1*int(self.surface.get_width()/2)

    def move(self):
        self.loc[0] -= 1

    def update(self):         
        self.rect = self.surface.get_rect(center = self.loc)
        SCREEN.blit(self.surface,self.rect) 

        
class Timer: # Times when to make API call
    def __init__(self, sec): # Make object with a set time to wait
        self.stop_time = sec

    def start(self): # Get the start time of the timer
        self.start_time = time.time()

    def timesup(self): # Check if time has elapsed
        if time.time() > (self.start_time + self.stop_time):
            return True
        else:
            return False

          
# Stocks
names_and_prices = csv_to_data()
stock_name = names_and_prices[::2]
stock_price = names_and_prices[1::2]
old_price = stock_price
print_index = 0
update_index = 0

# Make initial Window
display = []
if float(stock_price[print_index]) > float(old_price[print_index]):
    color = GREEN
else:
    color = RED    
            
new_text = '    ' + stock_name[print_index] + ': ' + stock_price[print_index]
display.append(window(new_text, color))
print_index += 1

# Timing Stuff
curr_time = time.time() # Loop Timout
now = datetime.now() 
current_time = now.strftime("%H:%M:%S")
print("Time = ", current_time)
api_timer = Timer(120) # Price update timer
api_timer.start()

# Prediction Stuff
F = open('FIFO', 'r')

while not EXIT:

    from_fifo = F.read() # Read Prediction
    if from_fifo:        # If there is something in the FIFO
         display.append(subwindow(from_fifo))
         
    for obj in display:
        obj.move()
        obj.update()
        if (type(obj).__name__ == 'window'):
            if (obj.loc[0] < obj.trigger) and obj.addElement:
                obj.addElement = False
                new_text = '    ' + stock_name[print_index] + ': ' + stock_price[print_index]
                if float(stock_price[print_index]) > float(old_price[print_index]):
                    color = GREEN
                else:
                    color = RED    
                display.append(window(new_text, color))
                print_index += 1
                if print_index >= len(stock_name):
                    print_index = 0

        if obj.loc[0] < obj.EXIT:
            display.remove(obj)

    
    if api_timer.timesup():
        now = datetime.now()
        current_time = now.strftime("%H:%M:%S")
        print("Time = ", current_time)
        old_price[update_index] = stock_price[update_index]
        stock_price[update_index] = getPrice(stock_name[update_index])
        api_timer.start()
        update_index += 1
        if update_index >= len(stock_name):
            update_index = 0
        
     
    pygame.display.flip()
    SCREEN.fill(BLACK)
    time.sleep(.005)

F.close()
GPIO.cleanup()
data_to_csv(stock_name, stock_price)
sys.exit()

              

# Prediction Reader (listener.py)
#! /usr/bin/python3

import socket
import os
import sys
import time

HOST = "0.0.0.0"  # Standard loopback interface address (localhost)
PORT = 65432  # Port to listen on (non-privileged ports are > 1023)

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind((HOST, PORT))
    while True:
        s.listen()
        conn, addr = s.accept()
        with conn:
            #print(f"Connected by {addr}")
            while True:
                data = conn.recv(1024)
                #print(data)
                if not data:
                    break
                F = open("FIFO", 'w', 1)
                F.write(str(data, 'UTF-8'))
                F.close()
        
s.close()
              

# -*- coding: utf-8 -*-
"""Stock_predict_project_ver.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1xI1eEefH53TWqQ4jy6LyhFeRm20eeKp1
"""

## import libraries
import pandas as pd # linear algebra
import numpy as np # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
from keras.models import Sequential
import matplotlib.patches as mpatches
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from pandas.api.types import CategoricalDtype
import os

"""#Reading the stock dataset


"""

df = pd.read_csv("/content/stock_data.csv") # Reading dataset

df["Date"] = pd.to_datetime(df["datetime"])
df.set_index("Date", inplace = True)

df.shape # looking at the shape. we have 30 rows and 5 columns

# We will estimate the next 2 days with the data we have, and to test this prediction, we extract the last 3 days of data from df.
df=df.iloc[::-1] #reverse the row order to place the most recent stocks at the bottom
Conclusion = df[-2:]
df = df[:-2]

df

data = df.copy()

df.describe() # looking at statistical info

df.shape #looking at the shape. we have 1689 rows and 6 columns

df.info() # looking at information

"""No null values found here."""

data

df.shape[0]

plt.figure(figsize = (20, 12))
x = np.arange(0, df.shape[0], 1)
plt.subplot(2,1,1)
#plt.plot(x, df.open.values, color = "red", label = "Open Stock Price")
plt.plot(x, df["close"].values, color = "blue", label = "Close Stock Price")
plt.title("Stock Prices of the recent 100days ")
plt.xlabel("Days")
plt.ylabel("Stock Prices in US Dollar")
plt.legend(loc = "best")
plt.grid(which = "major", axis = "both")

#Close equate to df because we want to guess.

df = df[["close"]]
df.head()

# Time series visualization of Close

plt.figure(figsize=(20,9))
plt.plot(df["close"])
plt.title("Time Series")
plt.xlabel("Date")
plt.ylabel("Stock closing price prediction")
plt.show()

df

df.iloc[:, 0]

#We compress the data between -1 and 1 using MinMaxScaler.

scaler = MinMaxScaler(feature_range=(-1,1))
sc = scaler.fit_transform(df)

df.shape

timestep = 1 # We will forecast the next 3 days using the data of the last 5 days.

X = []
Y = []

for i in range(len(sc) - (timestep)):  
    X.append(sc[i:i+timestep])
    Y.append(sc[i+timestep])
    
X = np.asanyarray(X)
Y = np.asanyarray(Y)

k = 70  # The k value is determined according to the available data. For our data consisting of 27 rows, we use 20 for train and the rest for testing.

Xtrain = X[:k,:,:]
Xtest = X[k:,:,:]
Ytrain = Y[:k]
Ytest = Y[k:]

len(X)

# The shape of Xtrain and Xtest

print(Xtrain.shape)
print(Xtest.shape)

print(Ytrain.shape)
print(Ytest.shape)

df.isnull().sum() # checking null values

"""# LSTM model training start

"""

from keras.layers import Dense, RepeatVector
from keras.layers import Flatten
from keras.layers import TimeDistributed
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D

# We should pay attention to the number of inputs and outputs during the construction of the LSTM model. We should have 5 inputs and 5 outputs.

epochs = 200
model = Sequential()

model.add(LSTM(64, activation = "relu", input_shape = (1,1), return_sequences = True))
model.add(LSTM(64, activation = "relu", return_sequences = True))
model.add(LSTM(32, activation = "sigmoid", return_sequences = False))
model.add(Dense(1))
model.compile(optimizer = "adam", loss="mse")

history = model.fit(Xtrain,Ytrain, epochs= epochs, batch_size = 32, verbose=0)

model.save("AMZN_model.h5")

import pickle
with open("AMZN_pickle_model", "wb") as file:
    pickle.dump(model, file)

import pickle
with open("AMZN_scaler", "wb") as file:
    pickle.dump(scaler,file)


model_filename = "My_AMZN_model.sav"
saved_model = pickle.dump(model, open(model_filename,'wb'))

print('Model is saved into to disk successfully Using Pickle')

plt.figure(figsize=(8,5))
plt.plot(history.history['loss'])
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Model Accuracy")

preds = model.predict(Xtest)
preds = scaler.inverse_transform(preds)

Xtest2 = [[0.6]]
pred2 = model.predict(Xtest2)

Ytest = np.asanyarray(Ytest)
Ytest = Ytest.reshape(-1,1)
Ytest =scaler.inverse_transform(Ytest)

Ytrain = np.asanyarray(Ytrain)
Ytrain = Ytrain.reshape(-1,1)
Ytrain = scaler.inverse_transform(Ytrain)

mean_squared_error(Ytest,preds)

# Graphing the test and predicted values for Close.

plt.figure(figsize=(20,9))
plt.plot(Ytest[:,0], 'blue', linewidth=3)
plt.plot(preds[:,0],'r' , linewidth=2)
plt.legend(('Test','Predicted'))
plt.title("Stock closing price prediction")
plt.show()

"""##Stock Predictions on Close prices

"""

# we create new DataFrame separately for our prediction and test data.


test = pd.DataFrame(Ytest, columns = ["Close"])
pred = pd.DataFrame(preds, columns = ["Close_pred"])

def insert_end(Xin,new_input):
    #print ('Before: \n', Xin , new_input )
    for i in range(timestep-1):
        Xin[:,i,:] = Xin[:,i+1,:]
        #print(f"ilk giren değerler {Xin}")
    Xin[:,timestep-1,:] = new_input
  
    return Xin

# We add to the end we guessed.

from datetime import timedelta
future = 2
forecast = []
Xin = Xtest[-1:, : , :]
time = []
for i in range(future):
    out = model.predict(Xin, batch_size = 1)
    forecast.append(out)
    Xin = insert_end(Xin, out[0,0])
    time.append(pd.to_datetime(data.index[-1]) + timedelta(days = i + 1))

time

# We inverse_transform the prediction outputs.

forecasted_output = np.asanyarray(forecast)
forecasted_output = forecasted_output.reshape(-1,1)
forecasted_output = scaler.inverse_transform(forecasted_output) #back to actual value

np.asanyarray(forecast).reshape(-1,1)

forecasted_output

#Date file forecasted_output 

forecasted_output = pd.DataFrame(forecasted_output)
date = pd.DataFrame(time)
df_result = pd.concat([date,forecasted_output],axis=1)
df_result.columns = "Date","Predicted_Close"

#prin the predicted stock prices of next two days
df_result

df_result.set_index("Date", inplace = True)

#We combine conclusion with df_result and plot it.

df_result = pd.concat([df_result, Conclusion], axis=1)

def mape(y_true, preds): 
    y_true, preds = np.array(y_true), np.array(preds)
    return np.mean(np.abs((y_true - preds) / y_true)) * 100 #in percentage

mape(df_result["close"], df_result["Predicted_Close"])